Got tables from appkernel.ccr.buffalo.edu modw database.
# cleanup
dbDisconnect(con)
Error in dbDisconnect(con) : object 'con' not found
# renaming things to make it more straightforward...?
all_res <- all_res1
Error: object 'all_res1' not found
# doesn't apply for me
# this section appears to determine what sort of node/cpu it ran on? So not exactly similar for me
edge_CPU_L5520 <-c(
sprintf("cpn-d14-%02d", seq(2,23)),
sprintf("cpn-d14-%02d", seq(26,39)),
sprintf("cpn-d15-%02d", c(seq(1,32),seq(34,40))),
sprintf("cpn-d16-%02d", c(seq(2,14),seq(17,41))),
"cpn-d15-33",
"cpn-d16-15",
"cpn-d16-16"
)
nodes_CPU_L5520 <- rep(FALSE,times=nrow(all_res))
for(i in 1:nrow(all_res)){
#i<-1
m_nodes<-strsplit(trimws(gsub(";"," ",all_res$nodes[[i]]))," ")[[1]]
rigth_cpu <- TRUE
for(m_node in m_nodes){
#print(paste(m_node,m_node %in% edge_CPU_L5520))
if(!(m_node %in% edge_CPU_L5520)){
rigth_cpu=FALSE
}
}
#print(paste(i,rigth_cpu))
nodes_CPU_L5520[i]<-rigth_cpu
}
all_res$resource[nodes_CPU_L5520] <- "edge8core_ibm"
all_res$resource[all_res$resource=="edge"] <- "edge8core_dell"
all <- all_res %>% filter(app!="test") %>%
select(state,task_id,resource,app,nnodes,walltime,collected,body)
# replacing on a conditional
all_res$cpu_type[grepl("cpn-u25-39",all_res$nodes)] <- "CPU-GOLD-6130"
Warning message:
In UseMethod("as.list") :
call dbDisconnect() when finished working with a connection
# parses the body for all of the various metrics/info
metrics <- all %>% group_by(state,resource,app,nnodes,task_id) %>% do(get_metrics(.$body) )
Warning in .Internal(unique(x, incomparables, fromLast, nmax)) :
call dbDisconnect() when finished working with a connection
|====================== | 14% ~13 s remaining
|====================== | 14% ~13 s remaining
|======================= | 14% ~13 s remaining
|======================== | 15% ~13 s remaining
|======================== | 15% ~13 s remaining
|========================= | 16% ~13 s remaining
|========================== | 16% ~12 s remaining
|=========================== | 16% ~12 s remaining
|=========================== | 17% ~13 s remaining
|============================ | 17% ~13 s remaining
|============================ | 17% ~13 s remaining
|============================= | 18% ~13 s remaining
|============================= | 18% ~13 s remaining
|============================== | 18% ~13 s remaining
|============================== | 19% ~13 s remaining
|=============================== | 19% ~13 s remaining
|=============================== | 19% ~13 s remaining
|================================ | 20% ~13 s remaining
|================================ | 20% ~13 s remaining
|================================= | 20% ~13 s remaining
|================================= | 21% ~14 s remaining
|================================== | 21% ~14 s remaining
|================================== | 21% ~14 s remaining
|=================================== | 22% ~14 s remaining
|=================================== | 22% ~14 s remaining
|==================================== | 22% ~14 s remaining
|==================================== | 23% ~14 s remaining
|===================================== | 23% ~14 s remaining
|====================================== | 23% ~14 s remaining
|====================================== | 24% ~14 s remaining
|======================================= | 24% ~14 s remaining
|======================================= | 24% ~14 s remaining
|======================================== | 24% ~14 s remaining
|======================================== | 25% ~14 s remaining
|========================================= | 25% ~14 s remaining
|========================================= | 25% ~14 s remaining
|========================================== | 26% ~14 s remaining
|========================================== | 26% ~14 s remaining
|=========================================== | 26% ~14 s remaining
|=========================================== | 27% ~14 s remaining
|============================================ | 27% ~14 s remaining
|============================================ | 27% ~14 s remaining
|============================================= | 28% ~14 s remaining
|============================================= | 28% ~14 s remaining
|============================================== | 28% ~14 s remaining
|============================================== | 28% ~14 s remaining
|=============================================== | 29% ~14 s remaining
|=============================================== | 29% ~14 s remaining
|================================================ | 29% ~14 s remaining
|================================================ | 30% ~14 s remaining
|================================================= | 30% ~14 s remaining
|================================================= | 30% ~14 s remaining
|================================================== | 31% ~14 s remaining
|================================================== | 31% ~14 s remaining
|=================================================== | 31% ~14 s remaining
|=================================================== | 32% ~14 s remaining
|==================================================== | 32% ~14 s remaining
|===================================================== | 32% ~14 s remaining
|===================================================== | 33% ~14 s remaining
|====================================================== | 33% ~14 s remaining
|====================================================== | 33% ~14 s remaining
|======================================================= | 34% ~14 s remaining
|======================================================= | 34% ~13 s remaining
|======================================================== | 34% ~13 s remaining
|======================================================== | 35% ~13 s remaining
|========================================================= | 35% ~13 s remaining
|========================================================= | 35% ~13 s remaining
|========================================================== | 36% ~13 s remaining
|========================================================== | 36% ~13 s remaining
|=========================================================== | 36% ~13 s remaining
|=========================================================== | 36% ~13 s remaining
|============================================================ | 37% ~13 s remaining
|============================================================ | 37% ~13 s remaining
|============================================================= | 37% ~13 s remaining
|============================================================= | 38% ~13 s remaining
|============================================================== | 38% ~13 s remaining
|============================================================== | 38% ~13 s remaining
|=============================================================== | 39% ~13 s remaining
|=============================================================== | 39% ~13 s remaining
|================================================================ | 39% ~13 s remaining
|================================================================ | 40% ~13 s remaining
|================================================================= | 40% ~13 s remaining
|================================================================= | 40% ~13 s remaining
|================================================================== | 40% ~13 s remaining
|================================================================== | 41% ~13 s remaining
|=================================================================== | 41% ~13 s remaining
|=================================================================== | 41% ~13 s remaining
|==================================================================== | 42% ~12 s remaining
|===================================================================== | 42% ~12 s remaining
|===================================================================== | 42% ~12 s remaining
|====================================================================== | 43% ~12 s remaining
|====================================================================== | 43% ~12 s remaining
|======================================================================= | 43% ~12 s remaining
|======================================================================= | 44% ~12 s remaining
|======================================================================== | 44% ~12 s remaining
|======================================================================== | 44% ~12 s remaining
|========================================================================= | 45% ~12 s remaining
|========================================================================= | 45% ~12 s remaining
|========================================================================== | 45% ~12 s remaining
|========================================================================== | 46% ~12 s remaining
|=========================================================================== | 46% ~12 s remaining
|=========================================================================== | 46% ~12 s remaining
|============================================================================ | 47% ~12 s remaining
|============================================================================ | 47% ~12 s remaining
|============================================================================= | 47% ~11 s remaining
|============================================================================= | 48% ~11 s remaining
|============================================================================== | 48% ~11 s remaining
|============================================================================== | 48% ~11 s remaining
|=============================================================================== | 48% ~11 s remaining
|================================================================================ | 49% ~11 s remaining
|================================================================================ | 49% ~11 s remaining
|================================================================================= | 49% ~11 s remaining
|================================================================================= | 50% ~11 s remaining
|================================================================================== | 50% ~11 s remaining
|================================================================================== | 50% ~11 s remaining
|=================================================================================== | 51% ~11 s remaining
|==================================================================================== | 51% ~11 s remaining
|===================================================================================== | 52% ~10 s remaining
|===================================================================================== | 52% ~10 s remaining
|====================================================================================== | 53% ~10 s remaining
|======================================================================================= | 53% ~10 s remaining
|======================================================================================= | 54% ~10 s remaining
|======================================================================================== | 54% ~10 s remaining
|========================================================================================= | 54% ~10 s remaining
|========================================================================================== | 55% ~10 s remaining
|========================================================================================== | 55% ~9 s remaining
|=========================================================================================== | 56% ~9 s remaining
|============================================================================================ | 56% ~9 s remaining
|============================================================================================ | 57% ~9 s remaining
|============================================================================================= | 57% ~9 s remaining
|============================================================================================== | 57% ~9 s remaining
|============================================================================================== | 58% ~9 s remaining
|=============================================================================================== | 58% ~9 s remaining
|================================================================================================ | 59% ~9 s remaining
|================================================================================================= | 59% ~8 s remaining
|================================================================================================== | 60% ~8 s remaining
|================================================================================================== | 60% ~8 s remaining
|=================================================================================================== | 61% ~8 s remaining
|==================================================================================================== | 61% ~8 s remaining
|==================================================================================================== | 61% ~8 s remaining
|===================================================================================================== | 62% ~8 s remaining
|====================================================================================================== | 62% ~8 s remaining
|======================================================================================================= | 63% ~8 s remaining
|======================================================================================================= | 63% ~7 s remaining
|======================================================================================================== | 64% ~7 s remaining
|========================================================================================================= | 64% ~7 s remaining
|========================================================================================================== | 65% ~7 s remaining
|=========================================================================================================== | 65% ~7 s remaining
|=========================================================================================================== | 66% ~7 s remaining
|============================================================================================================ | 66% ~7 s remaining
|============================================================================================================= | 67% ~7 s remaining
|============================================================================================================= | 67% ~7 s remaining
|============================================================================================================== | 67% ~7 s remaining
|============================================================================================================== | 68% ~7 s remaining
|=============================================================================================================== | 68% ~7 s remaining
|=============================================================================================================== | 68% ~6 s remaining
|================================================================================================================ | 68% ~6 s remaining
|================================================================================================================ | 69% ~6 s remaining
|================================================================================================================= | 69% ~6 s remaining
|================================================================================================================= | 69% ~6 s remaining
|================================================================================================================== | 70% ~6 s remaining
|================================================================================================================== | 70% ~6 s remaining
|=================================================================================================================== | 70% ~6 s remaining
|=================================================================================================================== | 71% ~6 s remaining
|==================================================================================================================== | 71% ~6 s remaining
|===================================================================================================================== | 71% ~6 s remaining
|===================================================================================================================== | 72% ~6 s remaining
|====================================================================================================================== | 72% ~6 s remaining
|====================================================================================================================== | 72% ~6 s remaining
|======================================================================================================================= | 73% ~6 s remaining
|======================================================================================================================= | 73% ~6 s remaining
|======================================================================================================================== | 73% ~6 s remaining
|======================================================================================================================== | 74% ~5 s remaining
|========================================================================================================================= | 74% ~5 s remaining
|========================================================================================================================= | 74% ~5 s remaining
|========================================================================================================================== | 74% ~5 s remaining
|========================================================================================================================== | 75% ~5 s remaining
|=========================================================================================================================== | 75% ~5 s remaining
|=========================================================================================================================== | 75% ~5 s remaining
|============================================================================================================================ | 76% ~5 s remaining
|============================================================================================================================ | 76% ~5 s remaining
|============================================================================================================================= | 76% ~5 s remaining
|============================================================================================================================= | 77% ~5 s remaining
|============================================================================================================================== | 77% ~5 s remaining
|============================================================================================================================== | 77% ~5 s remaining
|=============================================================================================================================== | 78% ~5 s remaining
|=============================================================================================================================== | 78% ~5 s remaining
|================================================================================================================================ | 78% ~5 s remaining
|================================================================================================================================ | 78% ~5 s remaining
|================================================================================================================================= | 79% ~4 s remaining
|================================================================================================================================= | 79% ~4 s remaining
|================================================================================================================================== | 79% ~4 s remaining
|================================================================================================================================== | 80% ~4 s remaining
|=================================================================================================================================== | 80% ~4 s remaining
|=================================================================================================================================== | 80% ~4 s remaining
|==================================================================================================================================== | 81% ~4 s remaining
|==================================================================================================================================== | 81% ~4 s remaining
|===================================================================================================================================== | 81% ~4 s remaining
|===================================================================================================================================== | 82% ~4 s remaining
|====================================================================================================================================== | 82% ~4 s remaining
|====================================================================================================================================== | 82% ~4 s remaining
|======================================================================================================================================= | 83% ~4 s remaining
|======================================================================================================================================= | 83% ~4 s remaining
|======================================================================================================================================== | 83% ~4 s remaining
|======================================================================================================================================== | 83% ~4 s remaining
|========================================================================================================================================= | 84% ~3 s remaining
|========================================================================================================================================= | 84% ~3 s remaining
|========================================================================================================================================== | 84% ~3 s remaining
|========================================================================================================================================== | 85% ~3 s remaining
|=========================================================================================================================================== | 85% ~3 s remaining
|=========================================================================================================================================== | 85% ~3 s remaining
|============================================================================================================================================ | 86% ~3 s remaining
|============================================================================================================================================ | 86% ~3 s remaining
|============================================================================================================================================= | 86% ~3 s remaining
|============================================================================================================================================= | 87% ~3 s remaining
|============================================================================================================================================== | 87% ~3 s remaining
|=============================================================================================================================================== | 87% ~3 s remaining
|=============================================================================================================================================== | 88% ~3 s remaining
|================================================================================================================================================ | 88% ~3 s remaining
|================================================================================================================================================ | 88% ~3 s remaining
|================================================================================================================================================= | 88% ~2 s remaining
|================================================================================================================================================= | 89% ~2 s remaining
|================================================================================================================================================= | 89% ~2 s remaining
|================================================================================================================================================== | 89% ~2 s remaining
|================================================================================================================================================== | 90% ~2 s remaining
|=================================================================================================================================================== | 90% ~2 s remaining
|=================================================================================================================================================== | 90% ~2 s remaining
|==================================================================================================================================================== | 91% ~2 s remaining
|===================================================================================================================================================== | 91% ~2 s remaining
|===================================================================================================================================================== | 91% ~2 s remaining
|===================================================================================================================================================== | 91% ~2 s remaining
|====================================================================================================================================================== | 92% ~2 s remaining
|====================================================================================================================================================== | 92% ~2 s remaining
|======================================================================================================================================================= | 92% ~2 s remaining
|======================================================================================================================================================= | 93% ~2 s remaining
|======================================================================================================================================================== | 93% ~2 s remaining
|======================================================================================================================================================== | 93% ~1 s remaining
|========================================================================================================================================================= | 94% ~1 s remaining
|========================================================================================================================================================== | 94% ~1 s remaining
|========================================================================================================================================================== | 94% ~1 s remaining
|=========================================================================================================================================================== | 95% ~1 s remaining
|=========================================================================================================================================================== | 95% ~1 s remaining
|============================================================================================================================================================ | 95% ~1 s remaining
|============================================================================================================================================================ | 96% ~1 s remaining
|============================================================================================================================================================= | 96% ~1 s remaining
|============================================================================================================================================================= | 96% ~1 s remaining
|============================================================================================================================================================== | 96% ~1 s remaining
|============================================================================================================================================================== | 97% ~1 s remaining
|=============================================================================================================================================================== | 97% ~1 s remaining
|=============================================================================================================================================================== | 97% ~1 s remaining
|================================================================================================================================================================ | 98% ~1 s remaining
|================================================================================================================================================================ | 98% ~0 s remaining
|================================================================================================================================================================= | 98% ~0 s remaining
|================================================================================================================================================================= | 98% ~0 s remaining
|================================================================================================================================================================= | 99% ~0 s remaining
|================================================================================================================================================================== | 99% ~0 s remaining
|================================================================================================================================================================== | 99% ~0 s remaining
|=================================================================================================================================================================== |100% ~0 s remaining
|====================================================================================================================================================================|100% ~0 s remaining
# gets only the entries that are hpcc
all_hpcc <- all[all$app == "HPCC",]
all_hpcc$task_id <- as.numeric(all_hpcc$task_id)
metrics_hpcc <- metrics[metrics$app == "HPCC",]
# keeps the units of metrics intact
metrics_hpcc$metric_and_units <- paste(metrics_hpcc$metric, "(", metrics_hpcc$units,")", sep="")
remove(temp)
# makes a data frame with 21 columns, 25 rows (however many metrics there are)
temp <- data.frame(matrix(ncol=length(unique(metrics_hpcc$metric_and_units)), nrow=length(unique(metrics_hpcc$task_id))))
# sets up the name of the columns (based on the metrics gotten)
column_names = c("task_id", unique(metrics_hpcc$metric_and_units))
colnames(temp) <- column_names
Error in names(x) <- value :
'names' attribute [21] must be the same length as the vector [20]
full_gamess <- setup_data("GAMESS")
full_gamess <- setup_data("GAMESS")
full_hpcg <- setup_data("HPCG")
full_hpcc <- setup_data("HPCC")
full_hpcg <- setup_data("HPCG")
full_hpcc <- setup_data("HPCC")
full_ior <- setup_data("IOR")
Error in `$<-.data.frame`(`*tmp*`, "metric_and_units", value = "()") :
replacement has 1 row, data has 0
t.test(hpcc_singularity$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`, hpcc_bare_metal$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`)
Welch Two Sample t-test
data: hpcc_singularity$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)` and hpcc_bare_metal$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`
t = -3.6828, df = 207.04, p-value = 0.0002944
alternative hypothesis: true difference in means is not equal to 0
95 percent confidence interval:
-510.4233 -154.4815
sample estimates:
mean of x mean of y
19772.84 20105.30
ggplot(full_hpcc, aes(x = resource, y = walltime, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='centerwhole')
colnames(full_hpcc) <- gsub(" ","_",colnames(full_hpcc))
ggplot(full_hpcc, aes(x = resource, y = `MPI Random Access(MUpdate per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center', binwidth=0.1)
ggplot(full_hpcc, aes(x = resource, y = `Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance(MFLOP per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
ggplot(full_hpcc, aes(x = resource, y = `Average STREAM 'Add' Memory Bandwidth(MByte per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
ggplot(full_hpcc, aes(x = resource, y = `Average STREAM 'Copy' Memory Bandwidth(MByte per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
ggplot(full_hpcc, aes(x = resource, y = `Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
ggplot(full_hpcc, aes(x = resource, y = `High Performance LINPACK Efficiency(Percent)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
ggplot(full_hpcc, aes(x = resource, y = `Parallel Matrix Transpose (PTRANS)(MByte per Second)`, fill=cpu_type)) +
geom_dotplot(binaxis='y', stackdir='center')
# making sure to filter out any resources we aint using
metrics %<>% filter(resource %in% c("vortex_bare_metal","vortex_dock_sing"))
metrics$resource <- factor(
metrics$resource,
levels = unique(c("vortex_bare_metal","vortex_dock_sing")))
levels(metrics$resource) <- c("TestCluster","ProductionCluster")
metrics$nnodes <- factor(metrics$nnodes,levels = unique(c(1,2,4,8,16,metrics$nnodes)))
metrics$state_id <- metrics$state
metrics$state <- as.character(metrics$state_id)
metrics$state[metrics$state=="1"]<-"Before"
metrics$state[metrics$state=="2"]<-"After"
metrics$state <- factor(metrics$state,levels = c("Before","After"))
metrics %<>% arrange(collected)
#convert md performance should be ns/day was sim second per Day
j <- metrics$metric=="Molecular Dynamics Performance"
metrics$value[j] <- metrics$value[j]*1e9
metrics$units[j] <- "ns/day"
write.csv(metrics,"metrics.csv")
save(
list=c("metrics"),
file="metrics.Rdata"
)
metrics %<>% filter(app %in% c("HPCC","NAMD"))
metrics$app <- factor(as.character(metrics$app),levels = c("HPCC","NAMD"))
selected_metrics <- c(
"Wall Clock Time",
"Molecular Dynamics Simulation Performance",
"High Performance LINPACK Floating-Point Performance",
"Fast Fourier Transform (FFTW) Floating-Point Performance",
"Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance",
"Average STREAM 'Copy' Memory Bandwidth"
)
metrics %<>% filter(metric %in% selected_metrics)
metrics$metric <- factor(as.character(metrics$metric),levels = selected_metrics)
metrics$metric_long <- metrics$metric
levels(metrics$metric) <- c(
"Wall Time",
"MD Perf.",
"LINPACK",
"FFT",
"DGEMM",
"STREAM"
)
#convert md performance should be ns/day was sim second per Day
j <- metrics$metric=="Molecular Dynamics Performance"
metrics$value[j] <- metrics$value[j]*1e9
j <- metrics$metric=="LINPACK"
metrics$value[j] <- metrics$value[j]/1024
#metrics %>% group_by(metric) %>% summarise(units=last(units))
metrics$units <- metrics$metric
levels(metrics$units) <- c(
"s",
"ns/day",
"GFLOP/s",
"MFLOP/s",
"MFLOP/s",
"MB/s"
)
save(
list=c("metrics"),
file="metrics_selected.Rdata"
)
load(file="metrics_selected.Rdata")
#select only what is needed2
#only up to 30 points for Testing-Cluster
metrics %>% group_by(resource, app, nnodes, metric, state) %>%
mutate(id = row_number()) %>%
summarise(n=n(),avg=mean(value),units=last(units))
metrics %<>% group_by(resource, app, nnodes, metric, state) %>%
mutate(id = row_number()) %>% ungroup()
metrics %<>% filter(!(resource=="Testing-Cluster" & id > 30L))
#Sparcify After for UBHPC
metrics %>% filter(resource=="UB-HPC" & app == "NAMD") %>% group_by(resource, app, nnodes, metric, state) %>%
mutate(id = row_number()) %>%
summarise(n=n(),avg=mean(value),units=last(units))
metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "NAMD" & nnodes == 1L & id > 10)
set.seed(08122018)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))
metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "NAMD" & nnodes == 2L & id > 20)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 10L))]
metrics %<>% filter(!(task_id %in% remover_task_id))
metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "HPCC" & nnodes == 1L & id > 10)
set.seed(08122018)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))
metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "HPCC" & nnodes == 2L & id > 10)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))
metrics %>% filter(resource=="UB-HPC") %>% group_by(resource, app, nnodes, metric, state) %>%
mutate(id = row_number()) %>%
summarise(n=n(),avg=mean(value),units=last(units))
save(
list=c("metrics"),
file="metrics_selected2.Rdata"
)
#Read metrics bigger smaller
ak_metrics <- read.csv("ak_metrics.csv",header = F,col.names = c("app","metric","better"))
m_app<-ak_metrics$app[[1]]
for(i in 2:nrow(ak_metrics)){
if(ak_metrics$app[[i]]==""){
ak_metrics$app[[i]]=m_app
}else{
m_app<-ak_metrics$app[[i]]
}
}
ak_metrics$BiggerBetter <- ak_metrics$better !="S"
ak_metrics %<>% filter(metric!="" & app %in% unlist(app_conv,use.names = F)) %>% select(-better)
save(
list=c("ak_metrics"),
file="ak_metrics.Rdata"
)